# -*- coding: UTF-8 -*-
import os

import pandas as pd
from selenium import webdriver, common
from lxml import etree
from urllib import request


def parse_html(url, xpath_table):

    # urllib.request.Request(url, data=None, headers={}, method=None)
    # url = r'http://www.lagou.com/zhaopin/Python/?labelWords=label'
    # headers = {
    #     'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
    #                   r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
    #     'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
    #     'Connection': 'keep-alive'
    # }
    # req = request.Request(url, headers=headers)
    # page = request.urlopen(req).read()
    # page = page.decode('utf-8')
    # data = bytes(urllib.parse.urlencode({'word': 'hello'}), encoding='utf8')
    req = request.Request(url)
    req.add_header("User-Agent",
                   "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36")

    with request.urlopen(req, timeout=1) as f:
        if f.status == 200:
            tree = etree.HTML(f.read().decode('utf-8'))
            # 定位获取表格信息
            table = tree.xpath(xpath_table)
            # 将byte类型解码为str类型
            return etree.tostring(table[0], encoding='utf8').decode()
        else:
            print('爬取失败！')
            return 'spider failed!'





# 开放基金排行 - 混合基金 10000条 按自定义排行
# http://fund.eastmoney.com/data/fundranking.html#thh;c0;r;sqjzf;pn10000;ddesc;qsd20190302;qed20200302;qdii;zq;gg;gzbd;gzfs;bbzt;sfbb


def main():


    href = 'http://fund.eastmoney.com/data/fundranking.html#thh;c0;r;szzf;pn50;ddesc;qsd20190305;qed20200305;qdii;zq;gg;gzbd;gzfs;bbzt;sfbb'
    # http://fund.eastmoney.com/data/fundranking.html#tall;c0;r;szzf;pn50;ddesc;qsd20190306;qed20200306;qdii;zq;gg;gzbd;gzfs;bbzt;sfbb
    # http://fund.eastmoney.com/data/fundranking.html#thh;c0;r;szzf;pn50;ddesc;qsd20190306;qed20200306;qdii;zq;gg;gzbd;gzfs;bbzt;sfbb
    # http://fund.eastmoney.com/data/fundranking.html#thh;c0;r;szzf;pn50;ddesc;qsd20190305;qed20200305;qdii;zq;gg;gzbd;gzfs;bbzt;sfbb
    # 获取js动态加载内容
    # driver = webdriver.Chrome(executable_path=os.path.dirname(os.path.abspath('.')) + '/tools/chromedriver.exe')
    option = webdriver.ChromeOptions()
    option.add_argument('headless')
    # browser = webdriver.Chrome(chrome_options=option) # 这里是重点，增加一个参数即可实现在不打开浏览器的情况下完成系列操作
    driver = webdriver.Chrome(chrome_options=option, executable_path='../tools/chromedriver.exe')
    driver.get(href)
    driver.implicitly_wait(5)
    program = driver.page_source
    # print('program: ', program)
    df_tb = pd.read_html(program, attrs={'id': 'dbtable'}, encoding='utf-8', header=0)[0]
    print("df_tb: \n", df_tb)
    driver.close()

if __name__ == '__main__':
    main()